package com.g2room.test;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.life.GrapChangshiData;
import com.g2room.driving.grap.life.model.po.LifeSaying;
import com.g2room.driving.util.ImageUtil;
import com.g2room.myuall.grap.ky.model.po.GrapErr;
import com.g2room.myuall.grap.univ.po.Article;
import com.g2room.myuall.util.DbUtil;
import com.g2room.myuall.util.HtmlUtil;
import com.g2room.myuall.util.StringUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrapZhiChangData implements Runnable {
	//http://www.cwbol.com/
//	public String baseUrl = "http://www.cmiyu.com";
//	private String source = "全民养生网";
//	public String baseUrl = "http://www.360changshi.com/";
//	private String source = "360常识网";
	private static List<String> urls = new ArrayList<String>();
	private static List<Article> artiss = new ArrayList<Article>();

	private Article artis;
	public static void main(String[] args) throws Exception{
//		GrapZhiChangData.getSourceUrl();
		// 获取360常识网的方法
 		GrapZhiChangData gcd = new GrapZhiChangData();
// 		gcd.getLiuxingyuNav();
// 		gcd.getZhiChangNav();
// 		gcd.getMiyuNav();
 		

 		
		GrapZhiChangData.getErrUrl();
 		for(Article a : artiss) {
 			System.out.println(a.getSourceUrl());
 			if(a.getHot().equals("2")) {
 				gcd.getMiyuPage(a.getSourceUrl(), a.getState());
 			} else if(a.getHot().equals("3")) {
 				gcd.getMiyuList(a.getSourceUrl(), a.getState());
 			} else if(a.getHot().equals("4")) {
 				LifeSaying arti = gcd.getMiyuDetail(a.getSourceUrl());
 		        if(null != arti) {
 		        	List<LifeSaying> ls = new ArrayList<LifeSaying>();
 		        	ls.add(arti);
 		        	gcd.insertMiyuDb(ls);
 		        }
 			}
        }
	}

	

	public void getMiyuNav() {
		Document doc = getDoc("http://www.cmiyu.com", "1");
		if(null == doc) return;
		
		Elements els = null;
		List<String> purls = new ArrayList<String>();
//		els = doc.select(".header .center ul li a");
//		for(Element el : els) {
//			purls.add(el.attr("href"));
//		}
		purls.add("/new/");
		// 更多的
		els = doc.select(".miyuheader ul li.my_1 a");
		for(Element el : els) {
			purls.add(el.attr("href"));
		}
		System.out.println("===miyu数量：" + purls.size());
 		for(String purl : purls) {
 			getMiyuPage("http://www.cmiyu.com"+purl, 1);
 		}
	}

	/**
	 * 获取分页列表
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getMiyuPage(String url, int state) {
		// 获取分页信息
		Document doc = getDoc(url, "2");
		if(doc == null) return;
		List<String> listUrls = new ArrayList<String>();
		Elements els = doc.select(".pages ul li a");
		if(!url.equals("http://www.cmiyu.com/new/"))  listUrls.add(url);
		if(null != els && !els.isEmpty()) {
			// 获取第2个 my122.html
			String subUrl = els.get(0).attr("href").replace("2.html", "");
			String text = els.get(els.size() - 1).attr("href").replace(subUrl, "").replace(".html", "");
			try {
				// http://www.cmiyu.com/new/list_81_ 1399.html
				// http://www.cmiyu.com/day/my1071.html
				Integer i = Integer.parseInt(text);
				for(int j = 2; j<i+1; j++) {
					if(url.equals("http://www.cmiyu.com/new/") && j < 1427) continue;
					listUrls.add(url + subUrl + j+".html");
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(String listUrl : listUrls) {
        	sleep();
        	getMiyuList(listUrl, state);
        }
	}
	
	/**
	 * 获取列表列表
	 */
	public void getMiyuList(String url, int state) {
		System.out.println("list======"+url);
		// 获取分页信息
		Document doc = getDoc(url, "3");
		if(doc == null) return;
		Elements els = doc.select(".list ul li a");
		List<String> durls = new ArrayList<String>();
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				durls.add("http://www.cmiyu.com" +el.attr("href"));
			}
		}

		List<LifeSaying> artiss = new ArrayList<LifeSaying>();
        for(String durl : durls) {
			if(urls.contains(durl)) continue;
			urls.add(durl);
        	sleep();
        	LifeSaying ar = getMiyuDetail(durl);
        	if(null != ar) artiss.add(ar);
        }
        if(artiss.size() > 0) insertMiyuDb(artiss);
	}
	
	/**
	 * 获取列表列表
	 */
	public LifeSaying getMiyuDetail(String url) {
		// 获取分页信息
		Document doc = getDoc(url, "4");
		if(doc == null) return null;
		LifeSaying arti = new LifeSaying();
		Elements els = doc.select(".content .left .top .weizhi a");
		// 获取类别
		if(null != els && !els.isEmpty()) {
			arti.setTypeName("谜语");
			arti.setSubtypeName(els.get(1).text());
		}
		arti.setSourceUrl(url);
		els = doc.select(".content .left .top div h1");
		if(null != els && !els.isEmpty()) {
			arti.setTitle(els.first().text());
		}
		els = doc.select(".content .left .top div.zy");
		if(null != els && !els.isEmpty()) {
			arti.setSummary(els.first().text().replace("小贴士：", ""));
		}
		els = doc.select(".content .left .top div.md h3");
		if(null != els && !els.isEmpty()) {
			if(els.size() == 2) {
				arti.setConundrum(els.get(0).text().replace("谜面：", "").replace("问：", ""));
				arti.setAnswer(els.get(1).text().replace("谜底：", "").replace("答：", ""));
			}
		} else {
			els = doc.select(".content .left .top div.myjsmd");
			if(null != els && !els.isEmpty()) {
				arti.setSummary(els.first().html());
			}
		}
		return arti;
	}
	


	public void getZhiChangNav() {
		Document doc = getDoc("http://www.zcbkw.com/", "21");
		if(null == doc) return;
		
		Elements els = null;
		List<String> purls = new ArrayList<String>();
		els = doc.select(".newnav_main a");
		for(Element el : els) {
			if(el.attr("href").split("/").length == 3 && !el.attr("href").startsWith("http")) {
				purls.add(el.attr("href"));
			}
		}
		// 更多的
		els = doc.select("#more2");
		for(Element el : els) {
			if(!el.attr("href").startsWith("http")) {
				purls.add(el.attr("href"));
			}
		}
		System.out.println("===职场数量：" + purls.size());
 		for(String purl : purls) {
 			getZhiChangPage("http://www.zcbkw.com"+purl, 1);
 		}
	}

	/**
	 * 获取分页列表
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getZhiChangPage(String url, int state) {
		// 获取分页信息
		Document doc = getDoc(url, "22");
		if(doc == null) return;
		List<String> listUrls = new ArrayList<String>();
		Elements els = doc.select(".epages a");
		listUrls.add(url);
		if(null != els && !els.isEmpty()) {
			String text = els.get(els.size() - 1).attr("href");
			try {
				// /zcgl/faze/index_5.html
				// http://www.zcbkw.com/zcgl/faze/ index_2.html
				Integer i = Integer.parseInt(text.substring(text.lastIndexOf("_")+1, text.indexOf(".")));
				for(int j = 2; j<i+1; j++) {
					listUrls.add(url + "index_"+j+".html");
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(String listUrl : listUrls) {
        	getZhiChangList(listUrl, state);
        }
	}
	
	/**
	 * 获取列表列表
	 */
	public void getZhiChangList(String url, int state) {
		// 获取分页信息
		Document doc = getDoc(url, "23");
		if(doc == null) return;
		Elements els = doc.select(".left2 ul li a");
		List<String> durls = new ArrayList<String>();
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				durls.add("http://www.zcbkw.com"+el.attr("href"));
			}
		}

		List<Article> artiss = new ArrayList<Article>();
        for(String durl : durls) {
        	Article ar = getZhiChangDetail(durl);
        	if(null != ar) artiss.add(ar);
        }
        if(artiss.size() > 0) insertArtiDb(artiss);
	}
	
	/**
	 * 获取列表列表
	 */
	public Article getZhiChangDetail(String url) {
		// 获取分页信息
		Document doc = getDoc(url, "24");
		if(doc == null) return null;
		Article arti = new Article();
		Elements els = doc.select(".content1_L a");
		// 获取类别
		if(null != els && !els.isEmpty()) {
			arti.setCategoryName("职场");
			arti.setTypeName(els.get(1).text());
			arti.setClassName(els.get(2).text());
		}
		arti.setSourceUrl(url);
		els = doc.select(".content2 .left h1");
		if(null != els && !els.isEmpty()) {
			arti.setTitle(els.first().text());
		}
		els = doc.select(".content2 .left .lei .lei_L span");
		if(null != els && !els.isEmpty()) {
			// 发布：2017-02-21
			arti.setPublishTime(els.get(1).text());
		}
		els = doc.select("#news_main");
		if(null != els && !els.isEmpty()) {
			arti.setContent(HtmlUtil.delAllTag(els.first().html()));
			String t = els.first().text().trim();
			arti.setSummary(t.substring(0, Math.min(150, t.length())).replaceAll("　", ""));
			// 判断是否有图片，有则获取第一张图片作为封面
			els = els.first().getElementsByTag("img");
			if(null != els && !els.isEmpty()) {
				String src = els.first().attr("src");
				if(src.startsWith("http:")) {
					arti.setCover(src);
					// 获取宽度高度
					Map<Integer, Integer> map = ImageUtil.getImgWidth(arti.getCover());
					if(null != map) {
						for(int key : map.keySet()) {
							arti.setCoverWidth(key);
							arti.setCoverHeight(map.get(key));
						}
					}
				}
			}
		}
		return arti;
	}
	

	public void insertArtiDb(Article arti) {
		List<Article> as = new ArrayList<Article>();
		as.add(arti);
		insertArtiDb(as);
	}
	public void insertArtiDb(List<Article> artis) {
		String sql = "insert into life_knowledge_zc(category_name,type_name,content,source_url,class_name,source,title,publish_time,cover,cover_height,cover_width, state,summary, title) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = DbUtil.getConn("life_grap").prepareStatement(sql);
			for(Article cc : artis) {
				ps.setString(1, cc.getCategoryName());
				ps.setString(2, cc.getTypeName());
				ps.setString(3, cc.getContent());
				ps.setString(4, cc.getSourceUrl());
				ps.setString(5, cc.getClassName());
				ps.setString(6, cc.getSource());
				ps.setString(7, cc.getTitle());
				ps.setString(8, cc.getPublishTime());
				ps.setInt(10, cc.getCoverHeight());
				ps.setInt(11, cc.getCoverWidth());
				ps.setString(9, cc.getCover());
				ps.setInt(12, cc.getState());
				ps.setString(13, cc.getSummary());
				ps.setString(14, cc.getTitle());
				ps.addBatch();
			}
			ps.executeBatch();	
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setType("5");
			ge.setContent(JSON.toJSONString(artis));
			ge.setDescription("获取DB错误");
			insert2ErrDb(ge);
		
		} finally {
			if(null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}
	

	public void insertMiyuDb(List<LifeSaying> artis) {
		String sql = "insert into life_riddle(type_name,subtype_name,summary,conundrum,answer,source,source_url,title) values(?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = DbUtil.getConn("life_grap").prepareStatement(sql);
			for(LifeSaying cc : artis) {
				ps.setString(1, cc.getTypeName());
				ps.setString(2, cc.getSubtypeName());
				ps.setString(3, cc.getSummary());
				ps.setString(4, cc.getConundrum());
				ps.setString(5, cc.getAnswer());
				ps.setString(6, cc.getSource());
				ps.setString(7, cc.getSourceUrl());
				ps.setString(8, cc.getTitle());
				ps.addBatch();
			}
			ps.executeBatch();	
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setType("5");
			ge.setContent(JSON.toJSONString(artis));
			ge.setDescription("获取DB错误");
			insert2ErrDb(ge);
		
		} finally {
			if(null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public void insert2ErrDb(GrapErr ge) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getConn("life_grap").prepareStatement("insert into grap_err_zc(description,url,content,type) values(?,?,?,?)");
			grapPs.setString(1, ge.getDescription());
			grapPs.setString(2, ge.getUrl());
			grapPs.setString(3, ge.getContent());
			grapPs.setString(4, ge.getType());
			grapPs.execute();
			grapPs.close();
		} catch(Exception e) {
			e.printStackTrace();
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	@Override
	public void run() {
//		get360ChangshiPage(artis);
//		getPage(artis);
//        for(Article arti : artis) {
//        }
	}
	

	public static void getSourceUrl() {
		try {
			Connection connection = DbUtil.getConn("life_grap");
			Statement statement = connection.createStatement();
			String sql = "select source_url from life_riddle ";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				urls.add(rs.getString("source_url"));
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}

	public static void getErrUrl() {
		try {
			Connection connection = DbUtil.getConn("life_grap");
			Statement statement = connection.createStatement();
			String sql = "select url,id,type from grap_err_zc";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				Article arti = new Article();
				arti.setSourceUrl(rs.getString("url"));
				arti.setState(rs.getInt("id"));
				arti.setHot(rs.getString("type"));
				artiss.add(arti);
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public Document getDoc(String url, String errCode) {
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
			e.printStackTrace();
			System.out.println(url + "======" + e.getMessage());
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType(errCode);
			ge.setDescription("错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		return doc;
	}

	public void sleep() {
    	try {
    		Thread.sleep(Integer.parseInt(StringUtil.genRandomNum(1)) * 100 * 3);
    	} catch (Exception e) {
    	}
	}
}
